{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"precision-recall.png\" width=\"50%\" height=\"50%\">\n",
    "\n",
    "*<center><font color='gray'>kflu.github.io/2016/08/26/2016-08-26-visualizing-precision-recall</font></center>*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from numpy import trapz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"auc-case-predictions.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0.255509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>1</td>\n",
       "      <td>0.630273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>1</td>\n",
       "      <td>0.352507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>1</td>\n",
       "      <td>0.958512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>1</td>\n",
       "      <td>0.221029</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    actual  prediction\n",
       "5        0    0.255509\n",
       "97       1    0.630273\n",
       "99       1    0.352507\n",
       "67       1    0.958512\n",
       "71       1    0.221029"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#thresholds = list(np.array(list(range(0, 105, 1)))/100)\n",
    "thresholds = list(np.array(list(range(0, 1000+1, 1)))/1000)\n",
    "\n",
    "roc_point = []\n",
    "\n",
    "for threshold in thresholds:\n",
    "\n",
    "    tp = 0; fp = 0; fn = 0; tn = 0\n",
    "\n",
    "    for index, instance in df.iterrows():\n",
    "        actual = instance[\"actual\"]\n",
    "        prediction = instance[\"prediction\"]\n",
    "\n",
    "        if prediction >= threshold:\n",
    "            prediction_class = 1\n",
    "        else:\n",
    "            prediction_class = 0\n",
    "\n",
    "        if prediction_class == 1 and actual == 1:\n",
    "            tp = tp + 1\n",
    "        elif actual == 1 and prediction_class == 0:\n",
    "            fn = fn + 1\n",
    "        elif actual == 0 and prediction_class == 1: \n",
    "            fp = fp + 1\n",
    "        elif actual == 0 and prediction_class == 0:\n",
    "            tn = tn + 1\n",
    "\n",
    "    #print(tp, fp, fn, tn)\n",
    "\n",
    "    tpr = tp / (tp + fn)\n",
    "    fpr = fp / (tn + fp)\n",
    "\n",
    "    #print(tpr, fpr)\n",
    "    roc_point.append([tpr, fpr])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pivot = pd.DataFrame(roc_point, columns = [\"x\", \"y\"])\n",
    "pivot[\"threshold\"] = thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>threshold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>517</th>\n",
       "      <td>0.72</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476</th>\n",
       "      <td>0.76</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>679</th>\n",
       "      <td>0.42</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>551</th>\n",
       "      <td>0.66</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>586</th>\n",
       "      <td>0.56</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>747</th>\n",
       "      <td>0.34</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>418</th>\n",
       "      <td>0.80</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>622</th>\n",
       "      <td>0.52</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>375</th>\n",
       "      <td>0.84</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>582</th>\n",
       "      <td>0.56</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.582</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        x     y  threshold\n",
       "517  0.72  0.24      0.517\n",
       "476  0.76  0.28      0.476\n",
       "679  0.42  0.12      0.679\n",
       "551  0.66  0.24      0.551\n",
       "586  0.56  0.20      0.586\n",
       "747  0.34  0.04      0.747\n",
       "418  0.80  0.40      0.418\n",
       "622  0.52  0.16      0.622\n",
       "375  0.84  0.44      0.375\n",
       "582  0.56  0.20      0.582"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pivot.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>threshold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <td>0.74</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        x     y  threshold\n",
       "500  0.74  0.24        0.5"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pivot[pivot['threshold'] == 0.5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ROC Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,'true positive rate')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VPX1//HXIew7yCoQ2XdQMQiKVdwRVBCtS9WqdalW2/5qq0DBDaSg1lZt/aq4173KIgqKWkUUBcEtYZdNCPu+B0hyfn/MGEOYCUPIzcwk7+fjkQf33rkzcy6BOXM/n8/5fMzdERERASgX7wBERCRxKCmIiEgeJQUREcmjpCAiInmUFEREJI+SgoiI5FFSEBGRPEoKIiKSR0lBRETylI93AIerXr163rx583iHISKSVL7++uuN7l7/UOclXVJo3rw5s2fPjncYIiJJxcx+jOU8NR+JiEgeJQUREcmjpCAiInmUFEREJI+SgoiI5FFSEBGRPEoKIiKSJ7CkYGbPmdl6M5sT5XEzs8fMbLGZpZtZt6BiERGR2ARZvPYC8G/gP1EePw9oE/7pATwR/lNEpNSZ8O0qHpqykNVb93B07SrccW47BhzfhGETMnht5kpy3Ekx44oezbh/QBeaD5500GssH90v8DjN3YN7cbPmwLvu3jnCY08BU939tfD+QqC3u68p7DXT0tJcFc0ikkwmfLuKIeMy2LM/J+9YlQopdEutxfQlmw/rtYqaGMzsa3dPO9R58exTaAKszLefGT4mIlKqPDRl4QEJAWDP/pzDTgglIZ5JwSIci3jbYmY3mdlsM5u9YcOGgMMSESleq7fuiXcIMYvnhHiZQLN8+02B1ZFOdPcxwBgINR8FH5qIFJf2QyeTlfPzf9vKKcaCkX2jtplHO372P6byw/pdecfaNKjGh7f3jtpW3/We99m+9+dv5zUrpZB+X5+ocUY7/3DjjHS8Se0qrEqSxBDPPoV+wG1AX0IdzI+5+4mHek31KYgkj4IJobg1rFGR7Vk5B7XVe25uxPeNlhgKJoSSYkRpHilE0vYpmNlrwJdAOzPLNLPrzexmM7s5fMpkYCmwGHga+F1QsYhIfASZEADW7dgXsa0+2vtG++CPR0KAUEI4r3PDA45d3K1J1A/+khh9FFjzkbtfcYjHHbg1qPcXEUkGU+auo0GNSowY0JlzOzXKO14SCSCSpFtkR0QSU4+RH7Jux768/YY1KsYxmuRxWfdUBp/XnlpVKsQ7FEDTXIhIMSiYEICD9oPQsEZFqlRIOeBYlQopVE6JNLgx1KdwOMeDVrVCOUYN7JIwCQGUFESkGBRXAiisLb1Ng2oHHGvToBozh57NqIFdaFK7CkZolM+ogV1YMLLvQR/0hY0+Sr+vT8TzD7dtP9rxZaP6UqX8gR+3NSqWY96I8yKeH0+Bjj4KgkYfSWnVYvCkA0aiGLBsdL+IzTIzh55N6yGTyM73hPIGi0f1i3o8mmhDK6O9b6yvcSjxajMvaWu3ZXHX23P4cN46OjepyeiBXencpFaJxxHr6CMlBZEEUDAhFLdoieFwP8yjJQYlhYPl5jqvz1rJqMnz2ZeTy+1nt+X6U1pQPiU+DTSxJgV1NIskgKC/mmUX0xsUVzNRtDb/0mLZxl0MHpvOzGWbOanlUYwa2IXm9aod+okJQElBRAJVOcUiVjSXRvtzcnn6s6U88tEPVCpfjtEDu3BZ92aYJU8SVFIQkSOWYkZOhKboFCu9CaCgOau2cedb6cxbs50+nRoxvH8nGtSsHO+wDpuSgkgCKMp0B4ejfDF9UY1We3BFj2a8PGNFxOOl3Z59OTzy0SKe+XwZdatV5MmrutGnc+N4h1VkGpIqkgCWje530LTBRqgztuAHccMaFVk+ut9BH/TljajHo40+KmxoZaT3jTb66P4BXbiqZyop4WaSFDOu6pnK/QO6RDy/tPhi8Ub6PDqNp6Yt5ZcnNOWj209L6oQAGn0kInLYtu3ez98mz+eN2Ss55qiqjBrYhZNb1Yt3WIXS6CORODvcegFJDu9lrOHuiXPZvGsfvz2tJX86qy2VK8SnIjoIaj4SCUDBhAChYaGthxz+eH5JDOu2Z/Hbl2ZzyyvfUL96Jd6+tRdDzutQqhIC6E5BJBDR6gKKq15ASo57qAjtb5Pnsy87l0F92nPDL1pQIU5FaEFTUhARiWL5xl0MHpfOjKWb6dmyLqMGdqVFkhShFZWSgohIAdk5uTzz+TL++eEiKpYPzWR6WVozypVLniK0olJSEBHJZ86qbQwam87c1ds5p2NDRgzoTMMkLEIrKiUFEREga38Oj3z0A09/tpQ6VSvyxJXd6NO5UVJNUVEclBREpMz7cskmhoxLZ/mm3Vya1pShfTtSq2riLHxTkpQURI5QUaaNlsSwbc9+Rr83n9e+Wklq3aq8ckMPerVO7CK0oCkpiBwBJYTk9f6ctdz99hw27tzLTaeGitCqVCxdNQdFoaQgUoJK+8IyyWD9jizueXsu781ZS4fGNXn2mu50aVryK6ElKiUFkYAoASQWd+e/s1cyctJ8srJzuePcdtx0astSW4RWVEoKUiZEW4c42vErn/6S6Us25x3r1aruAfuSXH7ctIsh4zL4YskmTmxRl9EDu9CyfvV4h5WQlBSk1IvW7n84x5UQklN2Ti7Pfr6Mf360iArlyjHyos5c0T21TBShFZWSgkgA1HQUf3NXb2Pw2AwyVm3j7I4NGdG/M41qlZ0itKJSUhA5QkoAiSVrfw6P/e8Hnpq2lDpVK/D4r7rRt0vZK0IrKiUFESk1Zi7dxJBxGSzduItLTmjKsH4dqF018hKiEpmSgogkve1Z+xn93gJenbmCZnWr8PL1PTilTdkuQisqjcUSKWD56H70alX3gGMF9/OfK/H1wdy1nP2PT3n9qxXccEoLpvy/U5UQjkCgdwpm1gd4FEgBnnH30QUeTwVeBGqHzxns7pODjEniq/3QyWTl/LzSTOUUY8HIvhGHgL5y40lRh4xGc7gVxtFe65UbTzqs15GSt2HHXu6dOJdJGWto36gGY65O49hmteMdVtIz92CWgjKzFGARcDaQCcwCrnD3efnOGQN86+5PmFlHYLK7Ny/sddPS0nz27NmBxCzBKpgQjkSkD/OiTDmhb/rJx9158+tMRk6az559OfzhzNb89rRWKkI7BDP72t3TDnVekHcKJwKL3X1pOKDXgf7AvHznOFAzvF0LWB1gPBJnxZUQpOxasWk3fx2fweeLN9K9eR1GDexK6wYqQitOQSaFJsDKfPuZQI8C59wLfGBmvweqAWdFeiEzuwm4CSA1NbXYAxWRxJadk8vz05fz8IcLKV+uHCMGdObKE1WEFoQgk0Kk31bBr4pXAC+4+8NmdhLwkpl1dvfcA57kPgYYA6Hmo0CilSI5nL4AkaKYv2Y7g8amk565jbM6NGDEgM40rlUl3mGVWkEmhUygWb79phzcPHQ90AfA3b80s8pAPWB9gHFJMSmYECA0HUSyJITKKfqWmciy9ufwr49/4KlPl1KrSgX+dcXxnN+1sYrQAhZkz8wsoI2ZtTCzisDlwMQC56wAzgQwsw5AZWBDgDFJMSrKfEAFP4grp1jUIaDROoGLcjzS+y4Y2TfWsKWEfbVsM30f+4zHP1lC/+Oa8NHtp3HBsUcrIZSAwEYfAZhZX+ARQsNNn3P3kWY2HJjt7hPDI46eBqoTalq6090/KOw1NfoocWi0jxS3HeEitFdmrqBpnSr87aIunNq2frzDKhUSYfQR4ZqDyQWO3Z1vex7QK8gYpHhEqi8QKU4fzVvHsAlzWL8ji+tPacGfz2lL1YqadKGk6W9cDilSfYGGl0px2bgzVIT2bvoa2jWswZNXn8BxKkKLGyUFOaQgC86k7HJ3xn6zivsnzWP33hxuP7stN5/WiorlVYQWT0oKEgglACnMys2hIrTPfthI2jF1GH1xF1o3qBHvsAQlBSkgWYaTSnLKyXWen76Mhz9YRDmD4f07cVWPY1SElkCUFCSPEoIEacHa7Qwam8H3K7dyRvsG3D+gM0fXVhFaolHjncQkWn1BJGo6kvz2Zufw8AcLOf+xz1m5eTePXn4cz16TpoSQoHSnIDGJVuilBCCFmb18M4PGprNkwy4GHt+EYed3pG41rYSWyJQURKTY7cjaz4PvL+SlGT/SpHYVXriuO73bNYh3WBIDJQURKVYfL1jH0PFzWLs9i+t6Necv57SjWiV91CQL/aZEpFhs3LmX+96Zxzvfr6Ztw+o8fuXJdEutE++w5DAdMilYaAaqK4GW7j48vIRmI3f/KvDoRCThuTvjv13FiHfnsXNvNn86qy239FYRWrKK5U7h/4Bc4AxgOLADGAt0DzAuEUkCmVt289fxc5i2aAPdUmvzwMVdadNQRWjJLJak0MPdu5nZtwDuviU8FbaIlFE5uc6LXyzn7x8sxID7LuzE1T1VhFYaxJIU9ptZCuFV08ysPqE7BxEpgxat28Gdb6Xz3cqt9G5Xn5EXdaGJag5KjViSwmPAeKCBmY0ELgHuCjQqEUk4e7NzePyTJTwxdTHVK5XnkcuOo/9xWvimtDlkUnD3V8zsa0IrpBkwwN3nBx6ZiCSMr3/czKCxGSxev5MBxx3NXed35KjqleIdlgQgltFHL7n71cCCCMdEpBTbuTebh95fwH9m/EjjmpV5/rrunK4itFItluajTvl3wv0LJwQTjogkik8WrGfo+AzWbM/impOa85dz21FdRWilXtTfsJkNAf4KVDGz7YSajgD2AWNKIDYRiYNNO/cy/N15vP3dalo3qM5bN5/MCceoCK2siJoU3H0UMMrMRrn7kBKMSUTiwN2Z8N0qhr8TKkL745lt+N3prahUPiXeoUkJiqWjeYiZ1QHaAJXzHZ8WZGAiUnIyt+xm2IQ5TF24gePDRWhtVYRWJsXS0XwD8EegKfAd0BP4klCFs4gksZxc56Uvl/PglIUA3HNBR359UnNSVIRWZsXSa/RHQlNazHD3082sPXBfsGGJSNB+WLeDQWPT+WbFVk5rW5+RF3WmaZ2q8Q5L4iyWpJDl7llmhplVcvcFZtYu8MhEJBD7snP5v6mLefyTUBHaPy87lgHHNVERmgCxJYVMM6sNTAA+NLMtwOpgwxKRIHyzYguDx6azaN1OLjz2aO65QEVocqBYOpovCm/ea2afALWA9wONSkSK1a692Tw0ZSEvfrmcxjUr89y1aZzRvmG8w5IEVGhSMLNyQLq7dwZw909LJCoRKTZTF65n6Pg5rN62h6t7HsOdfdqrCE2iKvRfhrvnmtn3Zpbq7itKKigROXKbd+1jxLvzGP/tKlrVr8abvz2JtOZ14x2WJLhYvi40Buaa2VfArp8OuvuFgUUlcbF8dD+aD54U8bgkD3dn4verue+deWzfs58/nNGaW89orSI0iUksSaHIw0/NrA/wKJACPOPuoyOccylwL6H1Gr53918V9f3kyCkBJLdVW/cwbHwGnyzcwLHNavPAxV1o36hmvMOSJBJLR3OR+hHCE+c9DpwNZAKzzGyiu8/Ld04bYAjQK7yim6ZfjLMrn/6S6Us25+33alWXV248KY4RSSxyc52XZvzIg+8vINfhrvM7cu3JKkKTwxfkytonAovdfam77wNeB/oXOOdG4HF33wLg7usDjEcOoWBCAJi+ZDNXPv1lnCKSWCxev4NfPvUl90ycS7dj6vDBn07l+lNaKCFIkQQ5BKEJsDLffibQo8A5bQHMbDqhJqZ73V3DXeOkYEI41HGJr33ZuTz56RL+/fFiqlZK4eFfHsvAbipCkyMTU1IwsypAqrsvPIzXjvQv0yO8fxugN6G5lT4zs87uvrXA+98E3ASQmpp6GCGIlE7frtjC4LEZLFy3gwvCRWj1VIQmxSCWCfEuAP4OVARamNlxwPAYRh9lAs3y7Tfl4EroTEJzKu0HlpnZQkJJYlb+k9x9DOE1HNLS0gomFilEtD6C9kMnk5Xz819l5RR9u0wGu/dl8/cpi3j+i2U0qlmZZ69J48wOKkKT4hPLncK9hPoHpgK4+3dm1jyG580C2phZC2AVcDlQcGTRBOAK4AUzq0eoOWlpDK8tMYjWRxBp2Gn+BFFQr1Ya254Ipi3awF/HZ5C5ZQ9X9UxlUJ/21KhcId5hSSkTS1LIdvdth9tO6e7ZZnYbMIVQf8Fz7j7XzIYDs919Yvixc8xsHpAD3OHumw7vEiSaovQF9GpVV6OPEsyWXfsYMWke475ZRcv61Xjz5pPoriI0CUgsSWGOmf0KSAkPIf0D8EUsL+7uk4HJBY7dnW/bgdvDP5IAlAASh7vzTvoa7ps4l2179nPb6a257YzWVK6gIjQJTixJ4ffAUGAv8Cqhb/f3BxmUSFm3Ztseho2fw/8WrOfYprV4+YYedGisIjQJXixJoZ27DyWUGKQUU2dz/OXmOq/M/JEH3l9Idm4uw/p14LpeqjmQkhNLUviHmTUG3gRed/e5AcckJaByih00+mjByL5xjEgWr9/JkHHpzFq+hVNa1+NvF3Uh9SithCYlK5ZpLk43s0bApcAYM6sJvOHuakJKYkoAiWNfdi5PfbqEf328mCoVU3jokq5cckJTFaFJXMRUvObua4HHwovs3AncjfoVRI7Y9yu3MmhsOgvW7qBf18bce0En6tdQEZrETyzFax2Ay4BLgE2E5jD6c8BxiZRqu/dl848PFvHc9GXUr1GJp3+dxtkdVYQm8RfLncLzwGvAOe6utZlFjtDnP2xkyPh0Vm7ew5U9Uhl0XntqqghNEkQsfQo9SyIQiU20RXA0bUXi27p7H/dPms9bX2fSsl413ripJz1aHhXvsEQOEDUpmNl/3f1SM8vgwInsjFDdWdfAo5MDREoI0Y4XNm2FlCx3Z1LGGu6dOJetu/dz6+mt+P0ZbVSEJgmpsDuFP4b/PL8kApGSo9XVSs7abVkMmzCHj+avo0uTWvznNz3oeLSK0CRxRU0K7r4mvPk7dx+U/zEzewAYdPCzJNEoAcRHbq7z6lcreOC9BezPzeWvfdvzm14tKJ8S5LpWIkculo7mszk4AZwX4ZiIAEs27GTIuAy+WraZk1sdxaiBXTjmqGrxDkskJoX1KdwC/A5oaWbp+R6qAUwPOjA5cupsLln7c3IZM20pj/7vByqXL8eDF3fll2kqQpPkUti97KvABcDE8J8//Zzg7leVQGxSQLSmoOWj+x2UADRtRclKz9zKhf+ezkNTFnJWhwZ89OfTuLR7MyUESToWmr06wgNmNd19u5lFnLjd3eOycG9aWprPnj07Hm8tcpA9+3L4x4cLefbzZdSrXokRAzpzbqdG8Q5L5CBm9rW7px3qvML6FF4lNPLoa0JDUvN/5XGg5RFFKJLkpi/eyJBxGazYvJsrTkxl8HntqVVFRWiS3AobfXR++M8WJReOSOLbtns/IyfP47+zM2lRrxqv39STnipCk1IilrmPegHfufsuM7sK6AY84u4rAo9OJIG4O+/NWcvdb89ly+593NK7FX88U0VoUrrEMiT1CeBYMzuW0AypzwIvAacFGZhIIlm3PYu7Jszhg3nr6NykJi9c153OTWrFOyyRYhdLUsh2dzez/sCj7v6smV0TdGASWbS5jyQYubnO67NWMmryfPbl5DLkvPZcf4qK0KT0iiUp7DCzIcDVwC/MLAVQb1ocFDb3kRJD8Vu2cReDx6Yzc9lmTmoZKkJrXk9FaFK6xZIULgN+BfzG3deaWSrwULBhicTP/pxcnv5sKY989AOVypfjgYu7cGmaag6kbIhl6uy1ZvYK0N3Mzge+cvf/BB+aSMmbs2obd76Vzrw12+nTqRHD+3eiQc3K8Q5LpMTEMvroUkJ3BlMJ1Sr8y8zucPe3Ao6tTOt6z/ts35uTt1+zkka4BGnPvhwe+WgRz3y+jLrVKvLkVd3o07lxvMMSKXGxNB8NBbq7+3oAM6sPfAQoKQSkYEIADtqX4vPF4o0MGZ/Bj5t2c3n3Zgzp20FFaFJmxZIUyv2UEMI2UficSXKElABKxrbd+/nb5Pm8MXslxxxVlVdv7MHJrerFOyyRuIolKbxvZlMIrdMMoY7nycGFVLZEG1EkwXp/zhruensum3ft47enteRPZ7VVEZoIsXU032FmA4FTCPUpjHH38YFHVgYoIZS89duzuPvtubw/dy0dG9fk+WtVhCaSXyx3CgBfADlALjAruHBEguHuvDFrJSMnz2dfdi6D+rTnhl+0oIKK0EQOEMvooxuAu4GP+Xn00XB3fy7o4CQ2Klwr3PKNuxgyLoMvl26iZ8u6jBrYlRYqQhOJKJY7hTuA4919E4CZHUXozuGQScHM+gCPAinAM+4+Osp5lwBvEhrlVCoXS2g9ZBLZ+ZauKF/EOiglgNhl5+TyzOfL+OeHi6hYvhyjBnbhsrRmlCunIjSRaGJJCpnAjnz7O4CVh3pSeDqMxwmt8ZwJzDKzie4+r8B5NYA/ADNjDTrZFEwIwEH7sejVKuJ6RxLBnFXbGDQ2nbmrt3Nup4YM79+ZhipCEzmkWBpUVwEzzexeM7sHmAEsNrPbzez2Qp53IrDY3Ze6+z7gdaB/hPNGAA8CWYcZe9I43ASwfHS/gxJAr1Z1eeXGk4oxqtIpa38Oo99bQP/Hp7Nu+16euLIbT12dpoQgEqNY7hSWhH9+8nb4zxqHeF4TDryjyAR65D/BzI4Hmrn7u2b2l2gvZGY3ATcBpKamxhBy8ojWHKQEcPi+XLKJIePSWb5pN5emNWVo347UqqoiNJHDEcuQ1PuK+NqRGm7zvjObWTngn8C1McQwBhgDoTWaixhPkQybkMFrM1eS406KGVf0aMb9A7poCusEsm3Pfka/N5/XvlpJat2qvHJDD3q1VhGaSFHEOiS1KDKBZvn2mwKr8+3XADoDU8OzTzYCJprZhYnS2TxsQgYvz/h5gbkcd16eseKAY/lFm8K6vEVuQipqZ7P87P05a7n77Tls3LmXm04NFaFVqagiNJGiCnKQ9iygjZm1MLOKwOXAxJ8edPdt7l7P3Zu7e3NCfRUJkxAAXpt5yP70mCwe1e+gBFDeQselaNbvyOKWl7/m5pe/5qjqlXj71lP4a98OSggiRyiwOwV3zzaz24AphIakPufuc81sODDb3ScW/grxl+PF11KlBFA83J03Z2dy/6R5ZGXncse57bjp1JYqQhMpJrEUr7UltE5zQ3fvbGZdCX2jv/9Qz3X3yRSYJ8nd745ybu+YIi5BKWbFmhjkyPy4KVSE9sWSTZzYoi6jB3ahZf3q8Q5LpFSJ5evV08AQYD+Au6cTagoq9a7o0ezQJ0ngsnNyGTNtCec+Mo2MzG2MvKgzr9/YUwlBJACxNB9VdfevCixFmB1QPAnl/gFdADT6KI7mrt7G4LEZZKzaxtkdGzKif2ca1VLNgUhQYkkKG82sFeHhpOEpKdYEGlUCuX9Al7zkkJ8SQLCy9ufw2P9+4KlpS6lTtQKP/6obfbs00jrJIgGLJSncSqhGoL2ZrQKWAVcFGlUC0R1ByZu5dBNDxmWwdOMufnlCU4b260DtqhXjHZZImRBL8dpS4Cwzq0ZoFbYdh3pOaRFtvYNo9QhyZLZn7Wf0ewt4deYKmtWtwsvX9+CUNipCEylJsYw+urvAPgDuPjygmKQM+mDuWu56ew4bduzlhlNacPs5balaMcjaShGJJJb/dbvybVcGzgfmBxOOlDUbduzl3olzmZSxhvaNajDm6jSObVY73mGJlFmxNB89nH/fzP5OvspkkaJwd978OpORk+azZ18OfzmnLb89rZWK0ETirCj351WBlsUdiJQdKzbt5q/jM/h88Ua6N6/DqIFdad1ANQciiSCWPoUMfp7dNAWoD6g/QQ5bTq7z/PRl/P2DhZQvV44RAzpz5YmpWglNJIHEcqdwfr7tbGCdu5eJ4jUpPvPXbGfw2HS+z9zGWR0aMGJAZxrXqhLvsESkgEKTQnjNg0nu3rmE4pFSJmt/Dv/+eDFPfrqEWlUq8K8rjuf8ro1VhCaSoApNCu6ea2bfm1mqu0deREAkiq+WbWbwuHSWbtjFxd2aMqxfB+pUUxGaSCKLpfmoMTDXzL4i3/BUd78wsKgkqe3I2s8D7y/g5RkraFqnCv/5zYmc2rZ+vMMSkRjEkhSKuhynlEEfzVvHsAlzWL8ji+tPacGfVYQmklRi+d/a190H5T9gZg8AnwYTkiSjjTtDRWjvpq+hXcMaPHn1CRynIjSRpBNLUjgbGFTg2HkRjkkZ5O6M/WYV90+ax+69Ofz57FARWsXyKkITSUZRk4KZ3QL8DmhpZun5HqoBTA86sESwfHQ/zZJaiJWbQ0Von/2wkbRj6jD64i60blAj3mGJyBEo7E7hVeA9YBQwON/xHe6+OdCo4qD90Mlk5fy89GblFGPByL5KABH8VIT28AeLKGcwvH8nrupxjIrQREqBqEnB3bcB24ArSi6c+CiYEACycpz2QyezYGTfOEWVmBas3c6gsRl8v3IrZ7RvwP0DOnN0bRWhiZQWGhYCByWEQx0vi/Zmh4rQnpi6hJpVKvDo5cdx4bFHqwhNpJRRUpBDmr18M4PGprNkwy4GHt+EYed3pK6K0ERKpTKXFKKtpiYH27k3mwffX8BLM37k6FpVeOG67vRu1yDeYYlIgMpUUlBCiN3HC9YxdPwc1m7P4tqTm/OXc9pRrVKZ+uciUibpf7kcYNPOvdz3zjwmfr+atg2r8/iVJ9MttU68wxKREqKkIECoCG38t6sY8e48du7N5k9nteWW3ipCEylrlBSEzC27+ev4OUxbtIFuqbV54OKutGmoIjSRskhJoQzLyXVe/GI5f/9gIQbcd2Enru6pIjSRskxJoRCluZp50bod3PlWOt+t3ErvdvUZeVEXmqgITaTMCzQpmFkf4FFCazs/4+6jCzx+O3ADoWU+NwC/cfcfg4wpmtKcAPLbm53D458s4Ympi6leqTyPXHYc/Y9TEZqIhASWFMwsBXic0CyrmcAsM5vo7vPynfYtkObuu8MT8D0IXBZUTGXd1z9uZtDYDBav38mA447mrvM7clT1SvEOS0QSSJB3CicCi919KYCZvQ70B/KSgrt/ku/8GcBVAcYIAULOAAAOcklEQVRTZu3cm81D7y/gPzN+pHHNyjx/XXdOVxGaiEQQZFJoAqzMt58J9Cjk/OsJzcoqxeiThesZNn4Oq7ft4ZqTmvOXc9tRXUVoIhJFkJ8OkRqpI84wZ2ZXAWnAaVEevwm4CSA1NbW44ivVNu3cy4h35zHhu9W0blCdt24+mROOURGaiBQuyKSQCTTLt98UWF3wJDM7CxgKnObueyO9kLuPAcYApKWlaerSQrg7b3+3muHvzmNH1n7+eGYbfnd6KyqVT4l3aCKSBIJMCrOANmbWAlgFXA78Kv8JZnY88BTQx93XBxhLmZC5ZTfDJsxh6sINHB8uQmurIjQROQyBJQV3zzaz24AphIakPufuc81sODDb3ScCDwHVgTfDQyJXuPuFQcVUWuXkOi99uZwHpywE4J4LOvLrk5qToiI0ETlMgfY4uvtkYHKBY3fn2z4ryPcvC35Yt4NBY9P5ZsVWTmtbn5EXdaZpnarxDktEkpSGoSSpfdm5/N/UxTz+SagI7Z+XHcuA45qoCE1EjoiSQhL6ZsUWBo9NZ9G6nfQ/7mjuVhGaiBQTJYUksmtvNg9NWciLXy6ncc3KPHdtGme0bxjvsESkFFFSSBJTF65naLgI7eqex3Bnn/YqQhORYqdPlQS3edc+Rrw7j/HfrqJV/Wq8+duTSGteN95hiUgppaSQoNydid+vZvg789i2Zz9/OKM1t57RWkVoIhIoJYUEtHrrHoZNmMPHC9ZzbLPavHJxF9o3qhnvsESkDFBSSCC5uc7LM3/kgfcWkOtw1/kdufZkFaGJSMlRUkgQi9fvYNDYDL7+cQu/aFOPv13UhWZ1VYQmIiVLSSHO9mXn8uSnS/j3x4upWimFh395LAO7qQhNROJDSSGOvl2xhcFjM1i4bgcXHHs091zQkXoqQhOROFJSiIPd+7L5+5RFPP/FMhrVrMyz16RxZgcVoYlI/CkplLBpizbw1/EZZG75qQitHTUqV4h3WCIiQBlLCilm5PjBa/SklED7/ZZd+xgxaR7jvllFy/rVePPmk+iuIjQRSTBlKilESgiFHS8O7s476Wu4b+Jctu3Zz22nt+a2M1pTuYKK0EQk8ZSppFDSdwprtu1h2Pg5/G/Beo5tWouXb+hBh8YqQhORxFWmkkKFcpCTE/l4ccrNdV75agUPvLeA7NxchvXrwHW9WqgITUQSXplKClk5kZuJoh0visXrdzJkXDqzlm/hlNahIrTUo1SEJiLJoUwlhSDtz8nlqU+X8Nj/FlOlYgoPXdKVS05oqiI0EUkqSgrF4PuVWxk0Np0Fa3fQr2tj7r2gE/VrqAhNRJJPmUoKBkRqKCrqd/nd+7L5xweLeG76MurXqMTTv07j7I4qQhOR5FWmkkK0noOi9Ch8/sNGhoxPZ+XmPVzZI5VB57WnporQRCTJlamkUBy27t7H/ZPm89bXmbSsV403bupJj5ZHxTssEZFioaQQI3dnUsYa7p04l62793Pr6a34/RltVIQmIqWKkkIM1m7LYtiEOXw0fx1dmtTiP7/pQcejVYQmIqWPkkIhcnOdV8NFaPtzcxnatwPX9WpO+ZRirnYTEUkQSgpRLNmwkyHjMvhq2WZObnUUowZ24ZijqsU7LBGRQCkpFLA/J5cx05by6P9+oHL5cjx4cVd+maYiNBEpG5QU8knP3MqgsRnMX7Odvl0ace+FnWhQo3K8wxIRKTFKCmF/mzyfZz5bSr3qlXjq6hM4t1OjeIckIlLiAu0xNbM+ZrbQzBab2eAIj1cyszfCj880s+ZBxlOYMdOWcln3VD68/TQlBBEpswK7UzCzFOBx4GwgE5hlZhPdfV6+064Htrh7azO7HHgAuCyomArz+k096akiNBEp44K8UzgRWOzuS919H/A60L/AOf2BF8PbbwFnWpx6dJUQRESCTQpNgJX59jPDxyKe4+7ZwDbgoE9nM7vJzGab2ewNGzYEFK6IiASZFCJ94y8491ws5+DuY9w9zd3T6tevXyzBiYjIwYJMCplAs3z7TYHV0c4xs/JALWBzgDGJiEghgkwKs4A2ZtbCzCoClwMTC5wzEbgmvH0J8LG7F9/amAUsH93vsI6LiJQ1gY0+cvdsM7sNmAKkAM+5+1wzGw7MdveJwLPAS2a2mNAdwuVBxfMTJQARkegCLV5z98nA5ALH7s63nQX8MsgYREQkdpruU0RE8igpiIhIHiUFERHJo6QgIiJ5lBRERCSPkoKIiORRUhARkTwWYAFxIMxsA/BjMbxUPWBjMbxOstD1ll5l6VpB11tUx7j7ISePS7qkUFzMbLa7p8U7jpKi6y29ytK1gq43aGo+EhGRPEoKIiKSpywnhTHxDqCE6XpLr7J0raDrDVSZ7VMQEZGDleU7BRERKaDUJwUz62NmC81ssZkNjvB4JTN7I/z4TDNrXvJRFo8YrvV2M5tnZulm9j8zOyYecRaXQ11vvvMuMTM3s6QesRLL9ZrZpeHf8Vwze7WkYyxOMfx7TjWzT8zs2/C/6b7xiLM4mNlzZrbezOZEedzM7LHw30W6mXULLBh3L7U/hBb3WQK0BCoC3wMdC5zzO+DJ8PblwBvxjjvAaz0dqBreviVZrzXW6w2fVwOYBswA0uIdd8C/3zbAt0Cd8H6DeMcd8PWOAW4Jb3cElsc77iO43lOBbsCcKI/3Bd4jtK59T2BmULGU9juFE4HF7r7U3fcBrwP9C5zTH3gxvP0WcKaZWQnGWFwOea3u/om77w7vziC0bnayiuV3CzACeBDIKsngAhDL9d4IPO7uWwDcfX0Jx1icYrleB2qGt2tx8BrwScPdp1H4+vT9gf94yAygtpk1DiKW0p4UmgAr8+1nho9FPMfds4FtwFElEl3xiuVa87ue0DePZHXI6zWz44Fm7v5uSQYWkFh+v22BtmY23cxmmFmfEouu+MVyvfcCV5lZJqEVHn9fMqHFxeH+/y6yQJfjTACRvvEXHG4VyznJIObrMLOrgDTgtEAjClah12tm5YB/AteWVEABi+X3W55QE1JvQneBn5lZZ3ffGnBsQYjleq8AXnD3h83sJELrvXd299zgwytxJfY5VdrvFDKBZvn2m3LwLWbeOWZWntBtaGG3cYkqlmvFzM4ChgIXuvveEootCIe63hpAZ2CqmS0n1A47MYk7m2P9t/y2u+9392XAQkJJIhnFcr3XA/8FcPcvgcqE5gkqjWL6/10cSntSmAW0MbMWZlaRUEfyxALnTASuCW9fAnzs4Z6dJHPIaw03pzxFKCEkc3szHOJ63X2bu9dz9+bu3pxQH8qF7j47PuEesVj+LU8gNJgAM6tHqDlpaYlGWXxiud4VwJkAZtaBUFLYUKJRlpyJwK/Do5B6AtvcfU0Qb1Sqm4/cPdvMbgOmEBrN8Jy7zzWz4cBsd58IPEvotnMxoTuEy+MXcdHFeK0PAdWBN8N96Svc/cK4BX0EYrzeUiPG650CnGNm84Ac4A533xS/qIsuxuv9M/C0mf2JUFPKtUn6hQ4ze41Qs1+9cB/JPUAFAHd/klCfSV9gMbAbuC6wWJL071BERAJQ2puPRETkMCgpiIhIHiUFERHJo6QgIiJ5lBRERCSPkoIkNDP7g5nNN7NXCjmnt5klxFQWZnbhTzN6mtkAM+uY77Hh4eLBkoqlt5mdXFLvJ6VDqa5TkFLhd8B54QrdhBceP/9TjcQA4F1gXvixu4v7/cysfHjOrkh6AzuBL4r7faX00p2CJCwze5LQ1MkTzexPZnaimX0Rnj//CzNrF+E5p5nZd+Gfb82sRvj4HWY2KzwX/X1R3m+nmT1sZt+E15uoHz5+XHiCuXQzG29mdcLH/2A/r0/xevjYtWb27/A39AuBh8KxtDKzFyy0tsN5ZvbffO/b28zeCW+fY2ZfhmN408yqR4hzqpn9zcw+Bf5oZhdYaC2Qb83sIzNraKF1QW4G/hR+/1+YWX0zGxv+e5hlZr2O4NcjpVW85xHXj34K+wGWA/XC2zWB8uHts4Cx4e3ewLvh7XeAXuHt6oTuhs8hNPe+Efoi9C5waoT3cuDK8PbdwL/D2+nAaeHt4cAj4e3VQKXwdu3wn9fme94LwCX5Xv8FQlOplCc0RUO18PEngKsIzdszLd/xQcDdEeKcCvxfvv06/FyIegPwcHj7XuAv+c57FTglvJ0KzI/371c/ifej5iNJJrWAF82sDaEP8AoRzpkO/CPcBzHO3TPN7BxCieHb8DnVCU0UN63Ac3OBN8LbLwPjzKwWoQ/8T8PHXwTeDG+nA6+Y2QRC8w7FxENTOLwPXGBmbwH9gDsJzVrbEZgenoakIvBllJd5I992U+ANC82vXxGI1tR2FtDRfl4upKaZ1XD3HbHGLqWfkoIkkxHAJ+5+Ubh5ZGrBE9x9tJlNIjRPzIxwx64Bo9z9qcN8v0PNAdOP0IpZFwJ3mVmnw3jtN4BbCc23Ncvdd1jo0/pDd78ihufvyrf9L+Af7j7RzHoTukOIpBxwkrvvOYw4pYxRn4Ikk1rAqvD2tZFOMLNW7p7h7g8As4H2hCZV+81P7fNm1sTMGkR4ejlCzTsAvwI+d/dtwBYz+0X4+NXApxZar6GZu39C6Ft+bUJ3IPntIDSFdyRTCS2/eCM/f+ufAfQys9bhOKuaWdsoz88v/9/LNfmOF3z/D4Dbftoxs+NieG0pY5QUJJk8CIwys+mEZs6M5P+Z2Rwz+x7YA7zn7h8Qak//0swyCC27GunDehfQycy+Bs4g1H8AoQ/ah8wsHTgufDwFeDn8et8C//SDF7N5Hbgj3AHcKv8D7p5DqG/jvPCfuPsGQsnutfB7zSCU1A7lXkIz334GbMx3/B3gop86moE/AGnhjvF5hDqiRQ6gWVJFwsxsp7sfNNpHpCzRnYKIiOTRnYKIiOTRnYKIiORRUhARkTxKCiIikkdJQURE8igpiIhIHiUFERHJ8/8BJq4NDYcajKgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x23e97f552b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(pivot.y, pivot.x)\n",
    "plt.plot([0, 1])\n",
    "plt.xlabel('false positive rate')\n",
    "plt.ylabel('true positive rate')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AUC Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "auc = round(abs(np.trapz(pivot.x, pivot.y)), 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7918"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
